3 * Base class for exporting
5 * Copyright © 2003, 2005, 2006 Brion Vibber <brion@pobox.com>
6 * https://www.mediawiki.org/
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 * http://www.gnu.org/copyleft/gpl.html
30 use Wikimedia\Rdbms\ResultWrapper
;
31 use Wikimedia\Rdbms\IDatabase
;
34 * @ingroup SpecialPage Dump
37 /** @var bool Return distinct author list (when not returning full history) */
38 public $list_authors = false;
41 public $dumpUploads = false;
44 public $dumpUploadFileContents = false;
47 public $author_list = "";
51 const STABLE
= 4; // extension defined
67 /** @var DumpOutput */
71 * Returns the export schema version.
74 public static function schemaVersion() {
79 * If using WikiExporter::STREAM to stream a large amount of data,
80 * provide a database connection which is not managed by
81 * LoadBalancer to read from: some history blob types will
82 * make additional queries to pull source data while the
83 * main query is still running.
85 * @param IDatabase $db
86 * @param int|array $history One of WikiExporter::FULL, WikiExporter::CURRENT,
87 * WikiExporter::RANGE or WikiExporter::STABLE, or an associative array:
88 * - offset: non-inclusive offset at which to start the query
89 * - limit: maximum number of rows to return
90 * - dir: "asc" or "desc" timestamp order
91 * @param int $buffer One of WikiExporter::BUFFER or WikiExporter::STREAM
92 * @param int $text One of WikiExporter::TEXT or WikiExporter::STUB
94 function __construct( $db, $history = self
::CURRENT
,
95 $buffer = self
::BUFFER
, $text = self
::TEXT
) {
97 $this->history
= $history;
98 $this->buffer
= $buffer;
99 $this->writer
= new XmlDumpWriter();
100 $this->sink
= new DumpOutput();
105 * Set the DumpOutput or DumpFilter object which will receive
106 * various row objects and XML output for filtering. Filters
107 * can be chained or used as callbacks.
109 * @param DumpOutput &$sink
111 public function setOutputSink( &$sink ) {
112 $this->sink
=& $sink;
115 public function openStream() {
116 $output = $this->writer
->openStream();
117 $this->sink
->writeOpenStream( $output );
120 public function closeStream() {
121 $output = $this->writer
->closeStream();
122 $this->sink
->writeCloseStream( $output );
126 * Dumps a series of page and revision records for all pages
127 * in the database, either including complete history or only
128 * the most recent version.
130 public function allPages() {
131 $this->dumpFrom( '' );
135 * Dumps a series of page and revision records for those pages
136 * in the database falling within the page_id range given.
137 * @param int $start Inclusive lower limit (this id is included)
138 * @param int $end Exclusive upper limit (this id is not included)
139 * If 0, no upper limit.
140 * @param bool $orderRevs order revisions within pages in ascending order
142 public function pagesByRange( $start, $end, $orderRevs ) {
144 $condition = 'rev_page >= ' . intval( $start );
146 $condition .= ' AND rev_page < ' . intval( $end );
149 $condition = 'page_id >= ' . intval( $start );
151 $condition .= ' AND page_id < ' . intval( $end );
154 $this->dumpFrom( $condition, $orderRevs );
158 * Dumps a series of page and revision records for those pages
159 * in the database with revisions falling within the rev_id range given.
160 * @param int $start Inclusive lower limit (this id is included)
161 * @param int $end Exclusive upper limit (this id is not included)
162 * If 0, no upper limit.
164 public function revsByRange( $start, $end ) {
165 $condition = 'rev_id >= ' . intval( $start );
167 $condition .= ' AND rev_id < ' . intval( $end );
169 $this->dumpFrom( $condition );
173 * @param Title $title
175 public function pageByTitle( $title ) {
177 'page_namespace=' . $title->getNamespace() .
178 ' AND page_title=' . $this->db
->addQuotes( $title->getDBkey() ) );
182 * @param string $name
183 * @throws MWException
185 public function pageByName( $name ) {
186 $title = Title
::newFromText( $name );
187 if ( is_null( $title ) ) {
188 throw new MWException( "Can't export invalid title" );
190 $this->pageByTitle( $title );
195 * @param array $names
197 public function pagesByName( $names ) {
198 foreach ( $names as $name ) {
199 $this->pageByName( $name );
203 public function allLogs() {
204 $this->dumpFrom( '' );
211 public function logsByRange( $start, $end ) {
212 $condition = 'log_id >= ' . intval( $start );
214 $condition .= ' AND log_id < ' . intval( $end );
216 $this->dumpFrom( $condition );
220 * Generates the distinct list of authors of an article
221 * Not called by default (depends on $this->list_authors)
222 * Can be set by Special:Export when not exporting whole history
226 protected function do_list_authors( $cond ) {
227 $this->author_list
= "<contributors>";
230 $revQuery = Revision
::getQueryInfo( [ 'page' ] );
231 $res = $this->db
->select(
234 'rev_user_text' => $revQuery['fields']['rev_user_text'],
235 'rev_user' => $revQuery['fields']['rev_user'],
238 $this->db
->bitAnd( 'rev_deleted', Revision
::DELETED_USER
) . ' = 0',
246 foreach ( $res as $row ) {
247 $this->author_list
.= "<contributor>" .
249 htmlentities( $row->rev_user_text
) .
256 $this->author_list
.= "</contributors>";
260 * @param string $cond
261 * @param bool $orderRevs
262 * @throws MWException
265 protected function dumpFrom( $cond = '', $orderRevs = false ) {
266 # For logging dumps...
267 if ( $this->history
& self
::LOGS
) {
270 $hideLogs = LogEventsList
::getExcludeClause( $this->db
);
272 $where[] = $hideLogs;
274 # Add on any caller specified conditions
278 # Get logging table name for logging.* clause
279 $logging = $this->db
->tableName( 'logging' );
281 if ( $this->buffer
== self
::STREAM
) {
282 $prev = $this->db
->bufferResults( false );
284 $result = null; // Assuring $result is not undefined, if exception occurs early
286 $commentQuery = CommentStore
::getStore()->getJoin( 'log_comment' );
287 $actorQuery = ActorMigration
::newMigration()->getJoin( 'log_user' );
290 $result = $this->db
->select(
291 array_merge( [ 'logging' ], $commentQuery['tables'], $actorQuery['tables'], [ 'user' ] ),
292 [ "{$logging}.*", 'user_name' ] +
$commentQuery['fields'] +
$actorQuery['fields'],
295 [ 'ORDER BY' => 'log_id', 'USE INDEX' => [ 'logging' => 'PRIMARY' ] ],
297 'user' => [ 'JOIN', 'user_id = ' . $actorQuery['fields']['log_user'] ]
298 ] +
$commentQuery['joins'] +
$actorQuery['joins']
300 $this->outputLogStream( $result );
301 if ( $this->buffer
== self
::STREAM
) {
302 $this->db
->bufferResults( $prev );
304 } catch ( Exception
$e ) {
305 // Throwing the exception does not reliably free the resultset, and
306 // would also leave the connection in unbuffered mode.
313 } catch ( Exception
$e2 ) {
314 // Already in panic mode -> ignoring $e2 as $e has
318 // Putting database back in previous buffer mode
320 if ( $this->buffer
== self
::STREAM
) {
321 $this->db
->bufferResults( $prev );
323 } catch ( Exception
$e2 ) {
324 // Already in panic mode -> ignoring $e2 as $e has
328 // Inform caller about problem
333 $revOpts = [ 'page' ];
334 if ( $this->text
!= self
::STUB
) {
337 $revQuery = Revision
::getQueryInfo( $revOpts );
339 // We want page primary rather than revision
340 $tables = array_merge( [ 'page' ], array_diff( $revQuery['tables'], [ 'page' ] ) );
341 $join = $revQuery['joins'] +
[
342 'revision' => $revQuery['joins']['page']
344 unset( $join['page'] );
346 $fields = array_merge( $revQuery['fields'], [ 'page_restrictions' ] );
349 if ( $cond !== '' ) {
352 $opts = [ 'ORDER BY' => 'page_id ASC' ];
353 $opts['USE INDEX'] = [];
354 if ( is_array( $this->history
) ) {
355 # Time offset/limit for all pages/history...
357 if ( $this->history
['dir'] == 'asc' ) {
359 $opts['ORDER BY'] = 'rev_timestamp ASC';
362 $opts['ORDER BY'] = 'rev_timestamp DESC';
365 if ( !empty( $this->history
['offset'] ) ) {
366 $conds[] = "rev_timestamp $op " .
367 $this->db
->addQuotes( $this->db
->timestamp( $this->history
['offset'] ) );
370 if ( !empty( $this->history
['limit'] ) ) {
371 $opts['LIMIT'] = intval( $this->history
['limit'] );
373 } elseif ( $this->history
& self
::FULL
) {
374 # Full history dumps...
375 # query optimization for history stub dumps
376 if ( $this->text
== self
::STUB
&& $orderRevs ) {
377 $tables = $revQuery['tables'];
378 $opts['ORDER BY'] = [ 'rev_page ASC', 'rev_id ASC' ];
379 $opts['USE INDEX']['revision'] = 'rev_page_id';
380 unset( $join['revision'] );
381 $join['page'] = [ 'INNER JOIN', 'rev_page=page_id' ];
383 } elseif ( $this->history
& self
::CURRENT
) {
384 # Latest revision dumps...
385 if ( $this->list_authors
&& $cond != '' ) { // List authors, if so desired
386 $this->do_list_authors( $cond );
388 $join['revision'] = [ 'INNER JOIN', 'page_id=rev_page AND page_latest=rev_id' ];
389 } elseif ( $this->history
& self
::STABLE
) {
390 # "Stable" revision dumps...
391 # Default JOIN, to be overridden...
392 $join['revision'] = [ 'INNER JOIN', 'page_id=rev_page AND page_latest=rev_id' ];
393 # One, and only one hook should set this, and return false
394 if ( Hooks
::run( 'WikiExporter::dumpStableQuery', [ &$tables, &$opts, &$join ] ) ) {
395 throw new MWException( __METHOD__
. " given invalid history dump type." );
397 } elseif ( $this->history
& self
::RANGE
) {
398 # Dump of revisions within a specified range
399 $opts['ORDER BY'] = [ 'rev_page ASC', 'rev_id ASC' ];
401 # Unknown history specification parameter?
402 throw new MWException( __METHOD__
. " given invalid history dump type." );
405 if ( $this->buffer
== self
::STREAM
) {
406 $prev = $this->db
->bufferResults( false );
408 $result = null; // Assuring $result is not undefined, if exception occurs early
410 Hooks
::run( 'ModifyExportQuery',
411 [ $this->db
, &$tables, &$cond, &$opts, &$join ] );
414 $result = $this->db
->select(
422 # Output dump results
423 $this->outputPageStream( $result );
425 if ( $this->buffer
== self
::STREAM
) {
426 $this->db
->bufferResults( $prev );
428 } catch ( Exception
$e ) {
429 // Throwing the exception does not reliably free the resultset, and
430 // would also leave the connection in unbuffered mode.
437 } catch ( Exception
$e2 ) {
438 // Already in panic mode -> ignoring $e2 as $e has
442 // Putting database back in previous buffer mode
444 if ( $this->buffer
== self
::STREAM
) {
445 $this->db
->bufferResults( $prev );
447 } catch ( Exception
$e2 ) {
448 // Already in panic mode -> ignoring $e2 as $e has
452 // Inform caller about problem
459 * Runs through a query result set dumping page and revision records.
460 * The result set should be sorted/grouped by page to avoid duplicate
461 * page records in the output.
464 * streaming (non-buffered) queries, as long as it was made on a
465 * separate database connection not managed by LoadBalancer; some
466 * blob storage types will make queries to pull source data.
468 * @param ResultWrapper $resultset
470 protected function outputPageStream( $resultset ) {
472 foreach ( $resultset as $row ) {
473 if ( $last === null ||
474 $last->page_namespace
!= $row->page_namespace ||
475 $last->page_title
!= $row->page_title
) {
476 if ( $last !== null ) {
478 if ( $this->dumpUploads
) {
479 $output .= $this->writer
->writeUploads( $last, $this->dumpUploadFileContents
);
481 $output .= $this->writer
->closePage();
482 $this->sink
->writeClosePage( $output );
484 $output = $this->writer
->openPage( $row );
485 $this->sink
->writeOpenPage( $row, $output );
488 $output = $this->writer
->writeRevision( $row );
489 $this->sink
->writeRevision( $row, $output );
491 if ( $last !== null ) {
493 if ( $this->dumpUploads
) {
494 $output .= $this->writer
->writeUploads( $last, $this->dumpUploadFileContents
);
496 $output .= $this->author_list
;
497 $output .= $this->writer
->closePage();
498 $this->sink
->writeClosePage( $output );
503 * @param ResultWrapper $resultset
505 protected function outputLogStream( $resultset ) {
506 foreach ( $resultset as $row ) {
507 $output = $this->writer
->writeLogItem( $row );
508 $this->sink
->writeLogItem( $row, $output );